#======================================================================
#======================== 17K Spider  v0.05 ===========================
#======================================================================

# -*- coding:utf-8 -*-

import urllib.request,re,json,os,time,sys
from urllib.request import Request, urlopen
from urllib.error import URLError
import logging


# 日志功能

# 配置日志信息  
logging.basicConfig(level=logging.INFO,  
                    format='%(asctime)s %(message)s',  
                    datefmt='%Y-%m-%d %H:%M:%S',  
                    filename='17kspider.log' ,  
                    filemode='a')  
# 定义一个Handler打印INFO及以上级别的日志到sys.stderr  
console = logging.StreamHandler()  
console.setLevel(logging.INFO)  
# 设置日志打印格式  
formatter = logging.Formatter('[%(asctime)s] %(message)s')  
console.setFormatter(formatter)  
# 将定义好的console日志handler添加到root logger  
logging.getLogger('').addHandler(console)
print('======================================================================')
print('======================== 17K Spider  v0.05 ===========================')
print('======================================================================')

# 日志功能结束

CLOCKSUM = 0
# 定时器
def dur( op=None, clock=[time.time()] ):
        global CLOCKSUM
        if op != None:
                duration = time.time() - clock[0]
                logging.info('%s。耗时 %.6f秒；'  % (op, duration))
        else:
        # 第一次调用用计时
                CLOCKSUM = time.time()
        clock[0] = time.time()
# def gettime():
        # return time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))

def getbookinfo():
        global BOOKNAME
        bookpage = 'http://h5.17k.com/book/%s.html' %BOOKID;
        req = Request(bookpage)
        try:
                response = urlopen(req)
        except URLError as e:
                if hasattr(e, 'reason'):
                        logging.error('连接服务器失败，错误信息：',e.reason)
                elif hasattr(e, 'code'):
                        logging.error('服务器响应异常，错误码：',e.code)
        else:
                html = response.read().decode("utf-8")
                reg_title = r'<p class="title">(.*?)</p>'
                reg_description = r'<section class="description" style="display: block;">(.*?)<!-- 看过的人还喜欢-->'
                try:
                        # 查找标题
                        book_name = re.findall(reg_title,html,re.S)[0]
                except:
                        book_name = '';
                BOOKNAME = book_name.strip()
                # print('@%s@' %BOOKNAME)

def gettitle(page):
        listpage = 'http://h5.17k.com/h5/book/ajaxBookList.k?bookId=%s&page=%d' %(BOOKID,page);
        html = urllib.request.urlopen(listpage).read().decode("utf-8")
        data = json.loads(html)
        # print(type(data))
        return data

def getcontent(chapterid):
        textpage = 'http://h5.17k.com/chapter/%s/%s.html' %(BOOKID,chapterid);
        req = Request(textpage)
        try:
                response = urlopen(req)
        except URLError as e:
                if hasattr(e, 'reason'):
                        logging.error('连接服务器失败，错误信息：',e.reason)
                elif hasattr(e, 'code'):
                        logging.error('服务器响应异常，错误码：',e.code)
        else:
                # print("good!")
                html = response.read().decode("utf-8")
                # print(html[0:16])
                # VIP章节没有广告，页面DIV有点儿变化。
                reg = r'<div id="TextContent">(.*?)</div>';
                try:
                                # 查找正文
                                data = re.findall(reg,html,re.S)[0]
                except:
                                data = '';
                # 先把原有换行符去掉
                data = data.replace('\r\n','')
                # 更换缩进为空格
                # data = data.replace('\u3000','  ')
                # 去除table
                data = data.replace('\t','')
                # 去除非VIP章节 空白注释
                data = data.replace('<!---->','')
                # 更换<br> 为换行符
                data = data.replace('<br>','\r\n')
                # 去除标题h1标签并添加换行
                data = re.sub('<h1>(.*?)</h1>','\\1\r\n\r\n',data)
                # 去除非VIP章节 广告
                data = re.sub('<em>.*?</script>','',data)
                return data
                # print (data)
        
def writettxt(_m,_title,_content):
        global BOOKNAME,DIRNAME
        # 将目录检测放到写入前 没必要则不新建空白文件夹
        if not os.path.exists(r'save/%s/%s' %(BOOKNAME,DIRNAME)):
                os.mkdir(r'save/%s/%s' %(BOOKNAME,DIRNAME))

        fn = open('save/%s/%s/%s. %s.txt' %(BOOKNAME,DIRNAME,_m,_title),'wb')
        # str转bytes
        _content = bytes(_content, encoding = "utf8")  
        fn.write(_content)
        fn.close()

def fetch_by_page(page):
        # 声明全局变量
        global DIRNAME,TOTALPAGE,TOTALCHAPTER,BOOKNAME,WRITEMODE
        gg =gettitle(page)
        if TOTALPAGE == 999:
                # print(gg['totalPage'])
                TOTALPAGE = gg['totalPage']
        if TOTALCHAPTER == 0:
                TOTALCHAPTER = gg['totalChapter']
        # jinp 表示页面中卷的页数 默认为零 有新卷则+1 章节数对应的-1
        jinp = 0
        # skip 表示跳过文件个数
        skip = 0
        # 找到datas
        for i in gg['datas']:
                # 如果有volumeName 则是卷名，创建文件夹
                # print(i)
                if 'volumeName' in i:
                        jinp += 1
                        logging.info('新卷：%s' %i['volumeName'])
                        DIRNAME = i['volumeName']
                        # 将目录收尾空白去除掉 防止目录出错
                        DIRNAME = DIRNAME.strip()
                        _dir = r'save/%s/%s' %(BOOKNAME,DIRNAME);
                        if not os.path.exists(_dir):
                                os.mkdir(_dir)
                else:
                        _id = i['id']
                        _title = i['name']
                        # gg[datas].index(i) 获取在列表中的索引值
                        # +1 因为索引从0开始计算，-jinp表示减去本页中的卷个数
                        _m = gg['datas'].index(i)+1-jinp+ (page-1)* 50
                        # 为小数值填充0
                        if TOTALCHAPTER >= 1000:
                                if _m<10:
                                        _m = '000%d' %_m
                                elif _m<100:
                                        _m = '00%d' %_m
                                elif _m<1000:
                                        _m = '0%d' %_m
                        elif TOTALCHAPTER<1000 and TOTALCHAPTER >=100:
                                if _m<10:
                                        _m = '00%d' %_m
                                elif _m<100:
                                        _m = '0%d' %_m
                        elif TOTALCHAPTER<100 and TOTALCHAPTER >= 10:
                                if _m<10:
                                        _m = '0%d' %_m
                        # 当写入模式为0时 判断文件是否存在，不存在则下载
                        if WRITEMODE == 0:
                                if not os.path.exists('save/%s/%s/%s. %s.txt' %(BOOKNAME,DIRNAME,_m,_title)):
                                        # 获取内容
                                        _content = getcontent(_id)
                                        if _content == '':
                                                continue
                                        writettxt(_m,_title,_content)
                                else:
                                        skip+=1
                        # 写入模式为1时，一律从服务器获取最新数据
                        elif WRITEMODE == 1:
                                # 获取内容
                                _content = getcontent(_id)
                                if _content == '':
                                        continue
                                writettxt(_m,_title,_content)
                        # print(_m,_id,_title,_content)
        if skip>0:
                dur('当前完成%s/%d页：%s/%s章（跳过%s章）' %(page,TOTALPAGE,_m,TOTALCHAPTER,skip))
        else:
                dur('当前完成%s/%d页：%s/%s章' %(page,TOTALPAGE,_m,TOTALCHAPTER))

def _begin():
        logging.info("《%s》(书号:%s)抓取开始。。。" %(BOOKNAME,BOOKID))
# 如果文件目录不存在就新建一个
        if not os.path.exists(r'save/%s' %BOOKNAME):
                os.mkdir(r'save/%s' %BOOKNAME)
        # 计时开始
        dur()
        # range(1,4)表示1-3
        for p in range(1,TOTALPAGE+1):
                # 手动退出
                if p > TOTALPAGE:
                        break
                fetch_by_page(p)
                # print('执行完毕')

        logging.info("任务全部完成！总共耗时%0.6f秒！" %(time.time()-CLOCKSUM))

# 获取传入参数
argv = sys.argv

DIRNAME = 'unsorted';
if not os.path.exists('save/'):
        os.mkdir('save/')
# page = 1;
# 获取写入模式 0 则跳过 1则替换
try:
        if argv[2] == '-c':
                WRITEMODE = 1
        else:
                WRITEMODE = 0
except:
        WRITEMODE = 0

try:
        BOOKID = int(argv[1])
        # 随便给个初值，判断等于初值就重新加载覆盖之
        TOTALPAGE = 999
        TOTALCHAPTER = 0
        # BOOKNAME = argv[2]
        # TOTALPAGE = int(argv[3])
        # TOTALCHAPTER = int(argv[4])
        getbookinfo()

except: 
        logging.error('参数错误')
        sys.exit(0)
else:
        _begin()

